/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package com.seari.engine;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.Page;
import com.gargoylesoftware.htmlunit.SgmlPage;
import com.gargoylesoftware.htmlunit.TopLevelWindow;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebRequest;
import com.gargoylesoftware.htmlunit.html.BaseFrameElement;
import com.gargoylesoftware.htmlunit.html.DomNode;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import static com.gargoylesoftware.htmlunit.util.UrlUtils.resolveUrl;
import com.seari.bean.IndexDoc;
import com.seari.bean.PageFetcher;
import com.seari.bean.SubSite;
import com.seari.main.listener.VspiderListener;
import com.seari.service.RedisService;
import com.seari.utils.CommonTools;
import com.seari.utils.DateUtil;
import com.seari.utils.HashTool;
import com.seari.utils.HtmlTools;
import com.seari.utils.SpiderHelper;
import java.net.SocketException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

/**
 *
 * @author Rainbow
 */
@Component
@Scope("prototype")
public class RollPageSpider implements Runnable
{
    protected int mode = SpiderHelper.MODE_NORMAL;
    long processTime;
    protected Pattern urlPattern;
    protected VspiderListener listener;
    protected WebClient webClient;
    private long startTime = System.currentTimeMillis();
    protected int httpCode = 0;
    protected int gid = 0;
    protected SubSite mySite;
    public int pageCount;
    public String nextPageUrl = "";
    protected HtmlElement nextPageElement;
    protected SimpleDateFormat sdf = new SimpleDateFormat(DateUtil.DATE_FORMAT_YYYYMMDD2);
    TopLevelWindow listWindow;
    Logger logger = LoggerFactory.getLogger(RollPageSpider.class);
    
    @Autowired
    private RedisService redisService;

    public RollPageSpider()
    {
    }
    
    public RollPageSpider(SubSite site)
    {
        initWebClient(site);
    }
    
    public void initRollPageSpider(SubSite site)
    {
        initWebClient(site);
    }
    
    
    private void initWebClient(SubSite site)
    {
        initWebClient(site, site != null && site.jsEnabled);
    }

    private void initWebClient(SubSite site, boolean jsEnabled)
    {
        mySite = site;
        if (site != null && site.urlTrait != null)
        {
            urlPattern = Pattern.compile(site.urlTrait);
        }
        webClient = new WebClient(BrowserVersion.CHROME);
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setJavaScriptEnabled(jsEnabled);
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        webClient.getOptions().setTimeout(60000);
        webClient.setJavaScriptTimeout(15000);
    }

    public void setWebClient(WebClient webClient)
    {
        this.webClient = webClient;
    }
    
    public void setWebClientJsEnabled(boolean jsEnabled)
    {
        webClient.getOptions().setJavaScriptEnabled(jsEnabled);
    }

    @Override
    public void run() {
        prebuildSite();
        Set<String> finishedArticles = new HashSet<String>();
        try
        {
            while (pageCount == 0 ||
                    (StringUtils.isNotBlank(mySite.getxLPathNextPage()) && pageCount < mySite.getPageCount()) ||
                    (mySite.pageCount > 0 && pageCount < mySite.getPageCount()))
            {                
                Set<IndexDoc> titles = getArticleLinkSet();
                afterFilter(titles);//URL匹配、去重等处理
                pageCount++;
                for (IndexDoc doc : titles)
                {
                    if (StringUtils.isBlank(doc.url) || finishedArticles.contains(doc.url))
                    {
                        continue;
                    }
                    if(mySite.getNeedContent())
                    {
                        //获取文章内容，待实现
                    }else{
                        redisService.insertHashToSet(mySite.getTableName(), doc.getGuid());
                        redisService.insertTitleToHash(mySite.getTableName()+"_hash", doc.getGuid(), doc.getTitle() + SpiderHelper.TITLE_URL_SPLITSTR + doc.getUrl());
                    }
                    finishedArticles.add(doc.url);
                }
                afterOperation(titles);
            }

        } catch (Exception e)
        {
            e.printStackTrace();
        }finally{
            cleanUp();
        }
    }
    
    protected void prebuildSite()
    {
        if(mySite.siteUrl.contains("{"))
        {
            //处理URL中的开始日期,取上月1号
            Calendar calendar = Calendar.getInstance();
            calendar.add(Calendar.MONTH, -1);
            calendar.set(Calendar.DATE,1);
            Date starttime = calendar.getTime();
            mySite.setSiteUrl(mySite.getSiteUrl().replace("{start}", sdf.format(starttime)));
            //处理URL中的结束日期，取当前日期
            mySite.setSiteUrl(mySite.getSiteUrl().replace("{end}", sdf.format(new Date())));
        }
    }
    
    //过滤文章URL及标题
    protected void afterFilter(Set<IndexDoc> titles)
    {
        Iterator iterator = titles.iterator();
        outer:
        while (iterator.hasNext())
        {
            IndexDoc doc = (IndexDoc) iterator.next();
            //根据URL生成HASH GUID
            doc.setGuid(CommonTools.hashKeyForDisk(doc.getUrl()));
            
            //根据文章标题关键字进行过滤
            if(StringUtils.isNotBlank(mySite.getTitleContainWords()))
            {
                String[] words = mySite.getTitleContainWords().split("\\|");
                boolean containFlag = false;
                for (String word : words)
                {
                    if(doc.title != null  && doc.title.contains(word))
                    {
                        containFlag = true;
                        break;
                    }
                }
                //如果不包含关键字，删除此文章链接
                if(!containFlag)
                {
                    iterator.remove();
                    continue;
                }
            }
            
            //根据GUID进行文章链接过滤
            if(redisService.isHashExist(mySite.getTableName(), doc.getGuid()))
            {
                iterator.remove();
                continue;
            }
            
            //其他过滤器
            otherFilter(doc);
        }
    }
    
    protected void otherFilter(IndexDoc doc)
    {
        //do nothing
        //提供给子类进行实现
    }
    
    
    protected final Set<IndexDoc> getArticleLinkSet() throws Exception
    {
        Set<IndexDoc> articleLinks;
        try
        {
//            initRollNewsSpider(mySite);
            if (StringUtils.isNotBlank(mySite.titleLineXpath))
            {
                articleLinks = getArticleLink();
            }
            else
            {
//                articleLinks = guessArticleLink();
                articleLinks = getArticleLink();
            }
        }
        finally
        {
            //cleanUp();
        }
        return articleLinks;
    }
    
    public Set<IndexDoc> getArticleLink() throws Exception
    {
        Set<IndexDoc> articles = new HashSet<IndexDoc>();
        SgmlPage page = getArticlePage(1);
        if (page == null)
        {
            logger.warn("fail to open site({})", mySite.siteUrl);
            return articles;
        }
        else
        {
            httpCode = page.getWebResponse().getStatusCode();
        }
        //获取下一页按钮元素或超链接
        getNextPageLink(page);
        listWindow = (TopLevelWindow) page.getEnclosingWindow().getTopWindow();
        //HtmlElement titleLineArea = page.getFirstByXPath(mySite.titleLineXpath);
        List<HtmlElement> links = page.getByXPath(mySite.titleLineXpath);
        System.out.println("com.seari.engine.RollPageSpider.getArticleLink():"+ Arrays.toString(links.toArray()));
        for (HtmlElement a : links)
        {
            String title = a.asText();
            String url = null;
            if (StringUtils.isNotBlank(mySite.xLptitleXpath))
            {
                HtmlElement titleElement = a.getFirstByXPath(mySite.xLptitleXpath);
                if(null != titleElement)
                {
                    title = titleElement.asText();
                    url = titleElement.getAttribute("href");
                }
            }
            if (StringUtils.isBlank(url))
            {
                url = a.getAttribute("href");
            }
            //URL为空时，不进行处理
            if(StringUtils.isBlank(url))continue;
            String link = getLink(page, url);
            IndexDoc indexDoc = new IndexDoc();
            indexDoc.url = link;
            indexDoc.title = title;
            articles.add(indexDoc);
            //暂时没有处理二级域名情况
//            if (!urlPattern.matcher(link).matches())
//            {
//                myLogger.log(Level.CONFIG, "link can not match, the link:{0} and the urlPattern:{1}", new Object[]
//                {
//                    link, urlPattern
//                });
//                try
//                {
//                    HtmlPage pageContent = a.click();
//                    link = pageContent.getUrl().toString();
//                }
//                catch (SocketException e)
//                {
//                    myLogger.log(Level.WARNING, "SocketException: Connection reset,url:", link);
//                    continue;
//                }
//
//            }

            //暂时没有处理二级域名情况
//            if (StringUtils.isNotBlank(title) && urlPattern.matcher(link).matches())
//            {
//                myLogger.log(Level.CONFIG, "link pass sucess({0})", link);
//                WebArticle webArticle = new WebArticle();
//                if (StringUtils.isNotBlank(mySite.xLPathPostTime))
//                {
//                    String postTime = MiscTools.getXpathStringValue(a, mySite.xLPathPostTime);
//                    if (postTime != null)
//                    {
//                        try
//                        {
//                            if ((postTime.contains("下午") || postTime.contains("上午")) && (mySite.postTimeFormat != null && mySite.postTimeFormat.contains("aa")))
//                            {
//                                webArticle.tmPost = parseComplicateTime(postTime, new SimpleDateFormat(mySite.postTimeFormat, Locale.CHINA));
//                            }
//                            else
//                            {
//                                webArticle.tmPost = parseComplicateTime(postTime, new SimpleDateFormat(mySite.postTimeFormat, Locale.US));
//                            }
//                        }
//                        catch (Exception ex)
//                        {
//                            if (myLogger.isLoggable(Level.WARNING))
//                            {
//                                myLogger.log(Level.WARNING, this + " Parsing title line for post time failed:", ex);
//                            }
//                        }
//                    }
//                }
//                webArticle.title = title;
//                webArticle.url = link;
//                articles.add(webArticle);
//            }
//            else if (StringUtils.isBlank(title))
//            {
//                myLogger.log(Level.CONFIG, "link is null the link xml:{0}", a.asXml());
//            }
//            else if (!urlPattern.matcher(link).matches())
//            {
//                myLogger.log(Level.CONFIG, "link can not match, the link:{0} and the urlPattern:{1}", new Object[]
//                {
//                    link, urlPattern
//                });
//            }
        }
        logger.info("this website({}) get article size:{}",page.getUrl(),articles.size());
        if (articles.isEmpty())
        {
            logger.warn("{}can not success to parse page, the page xml:{}, the page urlPattern ({})",this,page.asXml(),urlPattern);
        }
        return articles;
    }
    
    protected String getLink(SgmlPage page, String url)
    {
        String link = "";
        try
        {
            link = resolveUrl(page.getUrl(), url);
        }
        catch (Exception e)
        {
            e.printStackTrace();
        }
        if (link.contains("#"))
        {
            link = link.substring(0, link.indexOf('#'));
        }
        if(urlPattern != null)
        {
            try
            {
                Matcher matcher = urlPattern.matcher(link);
                if(matcher.find())
                {
                    link = matcher.group();
                }
            } catch (Exception e)
            {
                e.printStackTrace();
                logger.warn("link match exception occured:{}. Pattern:{}",link,mySite.getUrlTrait());
            }
        }
        return link;
    }
    
    public void getNextPageLink(SgmlPage page)
    {
        if(StringUtils.isNotBlank(mySite.getxLPathNextPage()))
        {
            nextPageElement = (HtmlElement) page.getFirstByXPath(mySite.xLPathNextPage);
        }else if(mySite.needNextPageUrl)
        {
        	nextPageUrl = getNextPageByPageUrlParam();
        }
    }
    
    public String getNextPageByPageUrlParam()
    {
        String result = "";
        if(StringUtils.isNotBlank(mySite.getPageUrlParam()))
        {
            result = mySite.siteUrl + "&" + mySite.getPageUrlParam() + "=" + (pageCount + 1);
        }else{
            result = mySite.siteUrl + "&page=" + (pageCount + 1);
        }
        return result;
    }
    
    public SgmlPage getArticlePage(int multiple) throws Exception
    {
        SgmlPage page = null;
        if (pageCount == 0)
        {
            page = openPage(mySite.siteUrl, multiple);
        }
        else if (nextPageElement != null)
        {
            page = nextPage(nextPageElement, multiple);
        }
        else if(StringUtils.isNotBlank(nextPageUrl))
        {
        	page = openPage(nextPageUrl, multiple);
        }
        else
        {
            String nextPageUrl = mySite.siteUrl + "&page=" + (pageCount + 1);
            page = openPage(nextPageUrl, multiple);
        }
        return page;
    }
    
    
    /**
     * 通过点击下一页按钮获取下页页面对象
     * @param nextPageLink
     * @param multiple
     * @return SgmlPage
     * @throws Exception
     */
    private SgmlPage nextPage(HtmlElement nextPageLink, int multiple) throws Exception
    {
        SgmlPage page = null;
        int i = 1;
        while (i <= 5)
        {
            try
            {
                page = (SgmlPage) nextPageLink.click();
                if (mySite.jsEnabled)
                {
                    webClient.waitForBackgroundJavaScript(multiple * 5000L);
                }
                break;
            }
            catch (Exception e)
            {
                TimeUnit.SECONDS.sleep(2);
                logger.warn("open next page ({}) failed, try again ... ({}), ",mySite.siteUrl,e.getMessage());
            }
            finally
            {
                ++i;
            }
        }
        return page;
    }
    
    public void processArticleLink(PageFetcher pageFetcher) throws Exception
    {
        try
        {
            setContentPageJsEnable();
            if (mode == SpiderHelper.MODE_TEST)
            {
                HtmlPage sp = getArticlePage(pageFetcher);
                if (sp == null)
                {
                    return;
                }
                pageFetcher.doc.url = pageFetcher.articleDetailUrl;
                pageFetcher.doc.guid = generateUrlGuid(pageFetcher.doc.url);
                getArticleContent(pageFetcher, sp);
                return;
            }
        } finally
        {
            cleanUp();
        }
    }
    
    public void getArticleContent(PageFetcher pageFetcher, HtmlPage sp) throws Exception
    {
        logger.info("open url:{}",pageFetcher.doc.url);
        try
        {
            
        }finally{
            TopLevelWindow tw = (TopLevelWindow) sp.getEnclosingWindow().getTopWindow();
            if (tw != listWindow)
            {
                try
                {
                    tw.close();
                }
                catch (Exception e)
                {
                    //do nothing
                }
            }
        }
    }
    
    //INDEXDOC后续处理
    public void afterOperation(Set<IndexDoc> titles)
    {
        
    }
    
    //同步比对HASH与MAIL中的KEY，不存在则发送邮件
    public void syncMail()
    {
        logger.info("sync mail for {}",mySite.getTableName());
        Set<String> hkeys = redisService.getHashKeys(mySite.getTableName()+"_hash");
        if(hkeys != null && hkeys.size() > 0)
        {
            StringBuilder sb = new StringBuilder();
            StringBuilder sb2 = new StringBuilder(); //临时存放HASH
            for (String hkey : hkeys)
            {
                if(!redisService.isHashExist(mySite.getTableName() + "_mail", hkey))
                {
                    sb.append(redisService.getHashValue(mySite.getTableName()+"_hash", hkey));
                    sb.append("\r\n");
                    sb.append("<br />");
                    sb2.append(hkey);
                    sb2.append(",");
                }
            }
            //同步到REDIS
            redisService.insertHashToSet(mySite.getTableName()+"_mail", sb2.toString().split(","));
        }
    }
    
    public void setContentPageJsEnable()
    {
        if (mode == SpiderHelper.MODE_SPECIFIC)
        {
            webClient.getOptions().setJavaScriptEnabled(false);
        }
    }
    
     public HtmlPage getArticlePage(PageFetcher pageFetcher) throws Exception
    {
        if (mySite.refreshLimit > 0)
        {
            Thread.sleep(mySite.refreshLimit * 1000);
        }
        IndexDoc doc = pageFetcher.doc;
        HtmlPage sp = null;
        Page page = null;
        try
        {
            page = openPage(doc.url);
            doc.url = page.getUrl().toString();
        }
        catch (Exception e)
        {
            try
            {
                doc.url = HtmlTools.resolveUrl(doc.url);
                page = openPage(doc.url);
            }
            catch (Exception ee)
            {
                logger.warn("faild to open page(url:{} - gid:{}",doc.url,gid);
                throw ee;
            }
        }
        if (webClient.getOptions().isJavaScriptEnabled())
        {
            webClient.waitForBackgroundJavaScript(5000);
        }
        if (page instanceof HtmlPage)
        {
            sp = (HtmlPage) page;
        }
        if (sp == null)
        {
            logger.warn("cannot be cast html page {0}", doc.url);
            return null;
        }
        httpCode = sp.getWebResponse().getStatusCode();
        if (httpCode == 404)
        {
            sp.cleanUp();
            logger.warn("the page return 404 code url({0}) guid({1})",doc.url,doc.guid);
            return null;
        }
        if (StringUtils.isNotBlank(mySite.frameList))
        {
            BaseFrameElement window = sp.getFirstByXPath(mySite.frameList);
            if (window != null)
            {
                return (HtmlPage) window.getEnclosedPage();
            }
        }
        return sp;
    }
    
    public SgmlPage openPage(String url, int multiple) throws Exception
    {
        SgmlPage page = null;
        int i = 1;
        while (i <= 5)
        {
            try
            {
                WebRequest wr = new WebRequest(new URL(url));
                wr.setAdditionalHeader("Referer", mySite.siteUrl);
                page = (SgmlPage) webClient.getPage(wr);
                if (mySite.jsEnabled)
                {
                    webClient.waitForBackgroundJavaScript(multiple * 5000L);
                }
                break;
            }
            catch (Exception e)
            {
                TimeUnit.SECONDS.sleep(2);
                logger.warn("open ({}) failed, try again ... ({}), ",url, e.getMessage());
            }
            finally
            {
                ++i;
            }
        }
        return page;
    }
    
    public SgmlPage openPage(String url) throws Exception
    {
        return webClient.getPage(url);
    }
    
    private void cleanUp()
    {
        if (webClient != null)
        {
            webClient.close();
            webClient = null;
        }
    }

    private String generateUrlGuid(String url)
    {
        String guid;
        guid = HashTool.generateUrlGuid(url);
        return guid;
    }
    
    public String removeSlashInUrl(String url)
    {
        //将URL中所有的//替换成/
        while (url.contains("//"))
        {
            url = url.replace("//", "/");
        }
        //补完url中的http://与https://
        url = url.replace(":/", "://");
        return url;
    }

    private void spiderStarted()
    {
        startTime = System.currentTimeMillis();
        if (listener != null)
        {
            listener.spiderStarted(this);
        }
    }

    private void spiderError(Exception e)
    {
        if (listener != null)
        {
            listener.spiderError(this, e);
        }
    }

    private void spiderCompleted()
    {
        long diff = System.currentTimeMillis() - startTime;
        processTime = TimeUnit.MILLISECONDS.toMinutes(diff);
        if (listener != null)
        {
            listener.spiderCompleted(this);
        }
    }

    public VspiderListener getListener()
    {
        return listener;
    }

    public void setListener(VspiderListener listener)
    {
        this.listener = listener;
    }


}
